{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "## Exploring the SciQ dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {},
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: OPENAI_API_KEY=\n"
     ]
    }
   ],
   "source": [
    "%env OPENAI_API_KEY="
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {},
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bc181e31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zsh:1: no matches found: refuel-autolabel[openai]\r\n"
     ]
    }
   ],
   "source": [
    "!pip3 install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {},
   "source": [
    "## Run the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from autolabel import LabelingAgent\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('config_sciq.json', 'r') as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "acb4a3de-fa84-4b94-b17a-7a6fac892a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-12 13:29:49 langchain.embeddings.openai WARNING: Retrying langchain.embeddings.openai.embed_with_retry.<locals>._embed_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬──────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $6.685   </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 1998     </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.00335 </span>│\n",
       "└──────────────────────────┴──────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬──────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$6.685  \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m1998    \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.00335\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴──────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding bank customers support complaints and queries.\n",
      "Your job is to correctly classify the provided input example into one of the following categories.\n",
      "Categories:\n",
      "activate_my_card\n",
      "age_limit\n",
      "apple_pay_or_google_pay\n",
      "atm_support\n",
      "automatic_top_up\n",
      "balance_not_updated_after_bank_transfer\n",
      "balance_not_updated_after_cheque_or_cash_deposit\n",
      "beneficiary_not_allowed\n",
      "cancel_transfer\n",
      "card_about_to_expire\n",
      "card_acceptance\n",
      "card_arrival\n",
      "card_delivery_estimate\n",
      "card_linking\n",
      "card_not_working\n",
      "card_payment_fee_charged\n",
      "card_payment_not_recognised\n",
      "card_payment_wrong_exchange_rate\n",
      "card_swallowed\n",
      "cash_withdrawal_charge\n",
      "cash_withdrawal_not_recognised\n",
      "change_pin\n",
      "compromised_card\n",
      "contactless_not_working\n",
      "country_support\n",
      "declined_card_payment\n",
      "declined_cash_withdrawal\n",
      "declined_transfer\n",
      "direct_debit_payment_not_recognised\n",
      "disposable_card_limits\n",
      "edit_personal_details\n",
      "exchange_charge\n",
      "exchange_rate\n",
      "exchange_via_app\n",
      "extra_charge_on_statement\n",
      "failed_transfer\n",
      "fiat_currency_support\n",
      "get_disposable_virtual_card\n",
      "get_physical_card\n",
      "getting_spare_card\n",
      "getting_virtual_card\n",
      "lost_or_stolen_card\n",
      "lost_or_stolen_phone\n",
      "order_physical_card\n",
      "passcode_forgotten\n",
      "pending_card_payment\n",
      "pending_cash_withdrawal\n",
      "pending_top_up\n",
      "pending_transfer\n",
      "pin_blocked\n",
      "receiving_money\n",
      "Refund_not_showing_up\n",
      "request_refund\n",
      "reverted_card_payment?\n",
      "supported_cards_and_currencies\n",
      "terminate_account\n",
      "top_up_by_bank_transfer_charge\n",
      "top_up_by_card_charge\n",
      "top_up_by_cash_or_cheque\n",
      "top_up_failed\n",
      "top_up_limits\n",
      "top_up_reverted\n",
      "topping_up_by_card\n",
      "transaction_charged_twice\n",
      "transfer_fee_charged\n",
      "transfer_into_account\n",
      "transfer_not_received_by_recipient\n",
      "transfer_timing\n",
      "unable_to_verify_identity\n",
      "verify_my_identity\n",
      "verify_source_of_funds\n",
      "verify_top_up\n",
      "virtual_card_not_working\n",
      "visa_or_mastercard\n",
      "why_verify_identity\n",
      "wrong_amount_of_cash_received\n",
      "wrong_exchange_rate_for_cash_withdrawal\n",
      "\n",
      "You will answer with just the the correct output label and nothing else.\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Input: I would like to delete my account please.\n",
      "Output: terminate_account\n",
      "\n",
      "Input: Can you help me get rid of my account?\n",
      "Output: terminate_account\n",
      "\n",
      "Input: Help me cancel my transaction\n",
      "Output: cancel_transfer\n",
      "\n",
      "Input: I need to cancel a purchase I made.\n",
      "Output: request_refund\n",
      "\n",
      "Input: I want to change my personal details.\n",
      "Output: edit_personal_details\n",
      "\n",
      "Input: How can I withdraw money?\n",
      "Output: atm_support\n",
      "\n",
      "Input: How do I do a successful transfer to an account?\n",
      "Output: beneficiary_not_allowed\n",
      "\n",
      "Input: Why wasn't I able to transfer to another account?\n",
      "Output: beneficiary_not_allowed\n",
      "\n",
      "Input: Can my daughter open an account?\n",
      "Output: age_limit\n",
      "\n",
      "Input: I need to update my current address\n",
      "Output: edit_personal_details\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Input: I want to close my account\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "agent.plan('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-12 13:30:13.878 | INFO     | autolabel.labeler:run:141 - Task run already exists.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">There is an existing task with following details: <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'2287622123'</span> <span style=\"color: #808000; text-decoration-color: #808000\">created_at</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">datetime</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">.datetime</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2023</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">44</span>,\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">38</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">598625</span><span style=\"font-weight: bold\">)</span> <span style=\"color: #808000; text-decoration-color: #808000\">task_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'ab16ad85a971805380e6a24713f197f6'</span> <span style=\"color: #808000; text-decoration-color: #808000\">dataset_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'1d12283d38845b25982ede8f1d7092ae'</span> \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">current_index</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> <span style=\"color: #808000; text-decoration-color: #808000\">output_file</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'banking_test_labeled.csv'</span> <span style=\"color: #808000; text-decoration-color: #808000\">status</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">TaskStatus.ACTIVE:</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'active'</span><span style=\"font-weight: bold\">&gt;</span> <span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span> <span style=\"color: #808000; text-decoration-color: #808000\">metrics</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "There is an existing task with following details: \u001b[33mid\u001b[0m=\u001b[32m'2287622123'\u001b[0m \u001b[33mcreated_at\u001b[0m=\u001b[1;35mdatetime\u001b[0m\u001b[1;35m.datetime\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m2023\u001b[0m, \u001b[1;36m6\u001b[0m, \u001b[1;36m12\u001b[0m, \u001b[1;36m12\u001b[0m, \u001b[1;36m44\u001b[0m,\n",
       "\u001b[1;36m38\u001b[0m, \u001b[1;36m598625\u001b[0m\u001b[1m)\u001b[0m \u001b[33mtask_id\u001b[0m=\u001b[32m'ab16ad85a971805380e6a24713f197f6'\u001b[0m \u001b[33mdataset_id\u001b[0m=\u001b[32m'1d12283d38845b25982ede8f1d7092ae'\u001b[0m \n",
       "\u001b[33mcurrent_index\u001b[0m=\u001b[1;36m5\u001b[0m \u001b[33moutput_file\u001b[0m=\u001b[32m'banking_test_labeled.csv'\u001b[0m \u001b[33mstatus\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mTaskStatus.ACTIVE:\u001b[0m\u001b[39m \u001b[0m\u001b[32m'active'\u001b[0m\u001b[1m>\u001b[0m \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m \u001b[33mmetrics\u001b[0m=\u001b[3;35mNone\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluating the existing task<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluating the existing task\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples have been labeled so far.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m5\u001b[0m examples have been labeled so far.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────── </span>Last Annotated Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────── \u001b[0mLast Annotated Example\u001b[92m ──────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Prompt</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mPrompt\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding bank customers support complaints and queries.\n",
      "Your job is to correctly classify the provided input example into one of the following categories.\n",
      "Categories:\n",
      "activate_my_card\n",
      "age_limit\n",
      "apple_pay_or_google_pay\n",
      "atm_support\n",
      "automatic_top_up\n",
      "balance_not_updated_after_bank_transfer\n",
      "balance_not_updated_after_cheque_or_cash_deposit\n",
      "beneficiary_not_allowed\n",
      "cancel_transfer\n",
      "card_about_to_expire\n",
      "card_acceptance\n",
      "card_arrival\n",
      "card_delivery_estimate\n",
      "card_linking\n",
      "card_not_working\n",
      "card_payment_fee_charged\n",
      "card_payment_not_recognised\n",
      "card_payment_wrong_exchange_rate\n",
      "card_swallowed\n",
      "cash_withdrawal_charge\n",
      "cash_withdrawal_not_recognised\n",
      "change_pin\n",
      "compromised_card\n",
      "contactless_not_working\n",
      "country_support\n",
      "declined_card_payment\n",
      "declined_cash_withdrawal\n",
      "declined_transfer\n",
      "direct_debit_payment_not_recognised\n",
      "disposable_card_limits\n",
      "edit_personal_details\n",
      "exchange_charge\n",
      "exchange_rate\n",
      "exchange_via_app\n",
      "extra_charge_on_statement\n",
      "failed_transfer\n",
      "fiat_currency_support\n",
      "get_disposable_virtual_card\n",
      "get_physical_card\n",
      "getting_spare_card\n",
      "getting_virtual_card\n",
      "lost_or_stolen_card\n",
      "lost_or_stolen_phone\n",
      "order_physical_card\n",
      "passcode_forgotten\n",
      "pending_card_payment\n",
      "pending_cash_withdrawal\n",
      "pending_top_up\n",
      "pending_transfer\n",
      "pin_blocked\n",
      "receiving_money\n",
      "Refund_not_showing_up\n",
      "request_refund\n",
      "reverted_card_payment?\n",
      "supported_cards_and_currencies\n",
      "terminate_account\n",
      "top_up_by_bank_transfer_charge\n",
      "top_up_by_card_charge\n",
      "top_up_by_cash_or_cheque\n",
      "top_up_failed\n",
      "top_up_limits\n",
      "top_up_reverted\n",
      "topping_up_by_card\n",
      "transaction_charged_twice\n",
      "transfer_fee_charged\n",
      "transfer_into_account\n",
      "transfer_not_received_by_recipient\n",
      "transfer_timing\n",
      "unable_to_verify_identity\n",
      "verify_my_identity\n",
      "verify_source_of_funds\n",
      "verify_top_up\n",
      "virtual_card_not_working\n",
      "visa_or_mastercard\n",
      "why_verify_identity\n",
      "wrong_amount_of_cash_received\n",
      "wrong_exchange_rate_for_cash_withdrawal\n",
      "\n",
      "You will answer with just the the correct output label and nothing else.\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Input: I was charged for something I didn't expect\n",
      "Output: transfer_fee_charged\n",
      "\n",
      "Input: I have a 1 euro fee on my statement.\n",
      "Output: extra_charge_on_statement\n",
      "\n",
      "Input: Is their a fee for top ups?\n",
      "Output: top_up_by_card_charge\n",
      "\n",
      "Input: Is there a reason I was charged a fee for using my card as payment? I've never had an extra fee before.\n",
      "Output: card_payment_fee_charged\n",
      "\n",
      "Input: What is the transfer fee charge?\n",
      "Output: transfer_fee_charged\n",
      "\n",
      "Input: I see some fees for cash withdraw.\n",
      "Output: cash_withdrawal_charge\n",
      "\n",
      "Input: Why was there a fee for my transfer?\n",
      "Output: transfer_fee_charged\n",
      "\n",
      "Input: The rate of exchange for my card payment is incorrect\n",
      "Output: card_payment_wrong_exchange_rate\n",
      "\n",
      "Input: Can you please check if I was charged twice?\n",
      "Output: transaction_charged_twice\n",
      "\n",
      "Input: There's a refund missing from my statement\n",
      "Output: Refund_not_showing_up\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Input: There is an extra $1 charge\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Annotation</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mAnnotation\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extra_charge_on_statement\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Do you want to resume the task? <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">[y/n]</span>: </pre>\n"
      ],
      "text/plain": [
       "Do you want to resume the task? \u001b[1;35m[y/n]\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Cost: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels, df, metrics_list = agent.run('test.csv', max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d7645ab",
   "metadata": {},
   "source": [
    "### Compute confidence scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5fbc1264",
   "metadata": {},
   "outputs": [],
   "source": [
    "config[\"model\"][\"compute_confidence\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1998f5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "119e6f22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬──────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $6.684   </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 1998     </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.00335 </span>│\n",
       "└──────────────────────────┴──────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬──────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$6.684  \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m1998    \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.00335\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴──────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding bank customers support complaints and queries.\n",
      "Your job is to correctly classify the provided input example into one of the following categories.\n",
      "Categories:\n",
      "activate_my_card\n",
      "age_limit\n",
      "apple_pay_or_google_pay\n",
      "atm_support\n",
      "automatic_top_up\n",
      "balance_not_updated_after_bank_transfer\n",
      "balance_not_updated_after_cheque_or_cash_deposit\n",
      "beneficiary_not_allowed\n",
      "cancel_transfer\n",
      "card_about_to_expire\n",
      "card_acceptance\n",
      "card_arrival\n",
      "card_delivery_estimate\n",
      "card_linking\n",
      "card_not_working\n",
      "card_payment_fee_charged\n",
      "card_payment_not_recognised\n",
      "card_payment_wrong_exchange_rate\n",
      "card_swallowed\n",
      "cash_withdrawal_charge\n",
      "cash_withdrawal_not_recognised\n",
      "change_pin\n",
      "compromised_card\n",
      "contactless_not_working\n",
      "country_support\n",
      "declined_card_payment\n",
      "declined_cash_withdrawal\n",
      "declined_transfer\n",
      "direct_debit_payment_not_recognised\n",
      "disposable_card_limits\n",
      "edit_personal_details\n",
      "exchange_charge\n",
      "exchange_rate\n",
      "exchange_via_app\n",
      "extra_charge_on_statement\n",
      "failed_transfer\n",
      "fiat_currency_support\n",
      "get_disposable_virtual_card\n",
      "get_physical_card\n",
      "getting_spare_card\n",
      "getting_virtual_card\n",
      "lost_or_stolen_card\n",
      "lost_or_stolen_phone\n",
      "order_physical_card\n",
      "passcode_forgotten\n",
      "pending_card_payment\n",
      "pending_cash_withdrawal\n",
      "pending_top_up\n",
      "pending_transfer\n",
      "pin_blocked\n",
      "receiving_money\n",
      "Refund_not_showing_up\n",
      "request_refund\n",
      "reverted_card_payment?\n",
      "supported_cards_and_currencies\n",
      "terminate_account\n",
      "top_up_by_bank_transfer_charge\n",
      "top_up_by_card_charge\n",
      "top_up_by_cash_or_cheque\n",
      "top_up_failed\n",
      "top_up_limits\n",
      "top_up_reverted\n",
      "topping_up_by_card\n",
      "transaction_charged_twice\n",
      "transfer_fee_charged\n",
      "transfer_into_account\n",
      "transfer_not_received_by_recipient\n",
      "transfer_timing\n",
      "unable_to_verify_identity\n",
      "verify_my_identity\n",
      "verify_source_of_funds\n",
      "verify_top_up\n",
      "virtual_card_not_working\n",
      "visa_or_mastercard\n",
      "why_verify_identity\n",
      "wrong_amount_of_cash_received\n",
      "wrong_exchange_rate_for_cash_withdrawal\n",
      "\n",
      "You will answer with just the the correct output label and nothing else.\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Input: I would like to delete my account please.\n",
      "Output: terminate_account\n",
      "\n",
      "Input: Can you help me get rid of my account?\n",
      "Output: terminate_account\n",
      "\n",
      "Input: Help me cancel my transaction\n",
      "Output: cancel_transfer\n",
      "\n",
      "Input: I need to cancel a purchase I made.\n",
      "Output: request_refund\n",
      "\n",
      "Input: I want to change my personal details.\n",
      "Output: edit_personal_details\n",
      "\n",
      "Input: How can I withdraw money?\n",
      "Output: atm_support\n",
      "\n",
      "Input: How do I do a successful transfer to an account?\n",
      "Output: beneficiary_not_allowed\n",
      "\n",
      "Input: Why wasn't I able to transfer to another account?\n",
      "Output: beneficiary_not_allowed\n",
      "\n",
      "Input: Can my daughter open an account?\n",
      "Output: age_limit\n",
      "\n",
      "Input: I need to update my current address\n",
      "Output: edit_personal_details\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Input: I want to close my account\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "agent.plan('test.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "63c74705",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-12 13:32:29.275 | INFO     | autolabel.labeler:run:141 - Task run already exists.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">There is an existing task with following details: <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'2710642923'</span> <span style=\"color: #808000; text-decoration-color: #808000\">created_at</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">datetime</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">.datetime</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2023</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">45</span>,\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">233284</span><span style=\"font-weight: bold\">)</span> <span style=\"color: #808000; text-decoration-color: #808000\">task_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'b5f154b0fcd26fd7c4fbac6ea5beca8d'</span> <span style=\"color: #808000; text-decoration-color: #808000\">dataset_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'1d12283d38845b25982ede8f1d7092ae'</span> \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">current_index</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> <span style=\"color: #808000; text-decoration-color: #808000\">output_file</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'banking_test_labeled.csv'</span> <span style=\"color: #808000; text-decoration-color: #808000\">status</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">TaskStatus.ACTIVE:</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'active'</span><span style=\"font-weight: bold\">&gt;</span> <span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span> <span style=\"color: #808000; text-decoration-color: #808000\">metrics</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "There is an existing task with following details: \u001b[33mid\u001b[0m=\u001b[32m'2710642923'\u001b[0m \u001b[33mcreated_at\u001b[0m=\u001b[1;35mdatetime\u001b[0m\u001b[1;35m.datetime\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m2023\u001b[0m, \u001b[1;36m6\u001b[0m, \u001b[1;36m12\u001b[0m, \u001b[1;36m12\u001b[0m, \u001b[1;36m45\u001b[0m,\n",
       "\u001b[1;36m14\u001b[0m, \u001b[1;36m233284\u001b[0m\u001b[1m)\u001b[0m \u001b[33mtask_id\u001b[0m=\u001b[32m'b5f154b0fcd26fd7c4fbac6ea5beca8d'\u001b[0m \u001b[33mdataset_id\u001b[0m=\u001b[32m'1d12283d38845b25982ede8f1d7092ae'\u001b[0m \n",
       "\u001b[33mcurrent_index\u001b[0m=\u001b[1;36m5\u001b[0m \u001b[33moutput_file\u001b[0m=\u001b[32m'banking_test_labeled.csv'\u001b[0m \u001b[33mstatus\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mTaskStatus.ACTIVE:\u001b[0m\u001b[39m \u001b[0m\u001b[32m'active'\u001b[0m\u001b[1m>\u001b[0m \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m \u001b[33mmetrics\u001b[0m=\u001b[3;35mNone\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluating the existing task<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluating the existing task\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 1.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold          </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf               </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 3.715415506885223  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2.715415506885223  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.2             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 3       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2.6973048594667963 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2.3766050567851615 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴────────────────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold         \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf              \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m3.715415506885223 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m2.715415506885223 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.2            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m3      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m2.6973048594667963\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m2.3766050567851615\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴────────────────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples have been labeled so far.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m5\u001b[0m examples have been labeled so far.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────── </span>Last Annotated Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────── \u001b[0mLast Annotated Example\u001b[92m ──────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Prompt</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mPrompt\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding bank customers support complaints and queries.\n",
      "Your job is to correctly classify the provided input example into one of the following categories.\n",
      "Categories:\n",
      "activate_my_card\n",
      "age_limit\n",
      "apple_pay_or_google_pay\n",
      "atm_support\n",
      "automatic_top_up\n",
      "balance_not_updated_after_bank_transfer\n",
      "balance_not_updated_after_cheque_or_cash_deposit\n",
      "beneficiary_not_allowed\n",
      "cancel_transfer\n",
      "card_about_to_expire\n",
      "card_acceptance\n",
      "card_arrival\n",
      "card_delivery_estimate\n",
      "card_linking\n",
      "card_not_working\n",
      "card_payment_fee_charged\n",
      "card_payment_not_recognised\n",
      "card_payment_wrong_exchange_rate\n",
      "card_swallowed\n",
      "cash_withdrawal_charge\n",
      "cash_withdrawal_not_recognised\n",
      "change_pin\n",
      "compromised_card\n",
      "contactless_not_working\n",
      "country_support\n",
      "declined_card_payment\n",
      "declined_cash_withdrawal\n",
      "declined_transfer\n",
      "direct_debit_payment_not_recognised\n",
      "disposable_card_limits\n",
      "edit_personal_details\n",
      "exchange_charge\n",
      "exchange_rate\n",
      "exchange_via_app\n",
      "extra_charge_on_statement\n",
      "failed_transfer\n",
      "fiat_currency_support\n",
      "get_disposable_virtual_card\n",
      "get_physical_card\n",
      "getting_spare_card\n",
      "getting_virtual_card\n",
      "lost_or_stolen_card\n",
      "lost_or_stolen_phone\n",
      "order_physical_card\n",
      "passcode_forgotten\n",
      "pending_card_payment\n",
      "pending_cash_withdrawal\n",
      "pending_top_up\n",
      "pending_transfer\n",
      "pin_blocked\n",
      "receiving_money\n",
      "Refund_not_showing_up\n",
      "request_refund\n",
      "reverted_card_payment?\n",
      "supported_cards_and_currencies\n",
      "terminate_account\n",
      "top_up_by_bank_transfer_charge\n",
      "top_up_by_card_charge\n",
      "top_up_by_cash_or_cheque\n",
      "top_up_failed\n",
      "top_up_limits\n",
      "top_up_reverted\n",
      "topping_up_by_card\n",
      "transaction_charged_twice\n",
      "transfer_fee_charged\n",
      "transfer_into_account\n",
      "transfer_not_received_by_recipient\n",
      "transfer_timing\n",
      "unable_to_verify_identity\n",
      "verify_my_identity\n",
      "verify_source_of_funds\n",
      "verify_top_up\n",
      "virtual_card_not_working\n",
      "visa_or_mastercard\n",
      "why_verify_identity\n",
      "wrong_amount_of_cash_received\n",
      "wrong_exchange_rate_for_cash_withdrawal\n",
      "\n",
      "You will answer with just the the correct output label and nothing else.\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Input: I was charged for something I didn't expect\n",
      "Output: transfer_fee_charged\n",
      "\n",
      "Input: I have a 1 euro fee on my statement.\n",
      "Output: extra_charge_on_statement\n",
      "\n",
      "Input: Is their a fee for top ups?\n",
      "Output: top_up_by_card_charge\n",
      "\n",
      "Input: Is there a reason I was charged a fee for using my card as payment? I've never had an extra fee before.\n",
      "Output: card_payment_fee_charged\n",
      "\n",
      "Input: What is the transfer fee charge?\n",
      "Output: transfer_fee_charged\n",
      "\n",
      "Input: I see some fees for cash withdraw.\n",
      "Output: cash_withdrawal_charge\n",
      "\n",
      "Input: Why was there a fee for my transfer?\n",
      "Output: transfer_fee_charged\n",
      "\n",
      "Input: The rate of exchange for my card payment is incorrect\n",
      "Output: card_payment_wrong_exchange_rate\n",
      "\n",
      "Input: Can you please check if I was charged twice?\n",
      "Output: transaction_charged_twice\n",
      "\n",
      "Input: There's a refund missing from my statement\n",
      "Output: Refund_not_showing_up\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Input: There is an extra $1 charge\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Annotation</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mAnnotation\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extra_charge_on_statement\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Do you want to resume the task? <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">[y/n]</span>: </pre>\n"
      ],
      "text/plain": [
       "Do you want to resume the task? \u001b[1;35m[y/n]\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 1.0\n",
      "Actual Cost: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold          </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf               </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 3.715415506885223  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2.715415506885223  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.2             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 3       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2.6973048594667963 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2.3766050567851615 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴────────────────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold         \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf              \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m3.715415506885223 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m2.715415506885223 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.2            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m3      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m2.6973048594667963\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m2.3766050567851615\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴────────────────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels, df, metrics_list = agent.run('test.csv', max_items=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31250a7f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
